import os
import base64
import PIL
import mimetypes
import inspect
from typing import Union
from functools import wraps
from google import genai
from google.genai import types
from fastcore.all import *
from fastcore.docments import *all_model_types = {
"gemini-2.0-flash": "llm-vertex#gemini-2.0-flash",
"gemini-2.0-flash-001": "llm-vertex#gemini-2.0-flash",
"gemini-2.0-pro-exp-02-05": "llm#gemini-2.0-pro",
"gemini-2.0-flash-lite-preview-02-05": "llm#gemini-2.0-flash-lite",
"gemini-1.5-flash": "llm-vertex#gemini-1.5-flash",
"gemini-1.5-pro": "llm-vertex#gemini-1.5-pro",
"gemini-1.5-pro-002": "llm-vertex#gemini-1.5-pro",
"gemini-1.5-flash-8b": "llm#gemini-1.5-flash-8b",
"gemini-2.0-flash-thinking-exp-01-21": "llm-thinking#gemini-2.0-flash-thinking",
"imagen-3.0-generate-002": "imagen#imagen-3.0"
}
thinking_models = [m for m in all_model_types if "thinking" in all_model_types[m]]
imagen_models = [m for m in all_model_types if "imagen" in all_model_types[m]]
vertex_models = [m for m in all_model_types if "vertex" in all_model_types[m]]
models = [m for m in all_model_types if "llm" in all_model_types[m]]
models['gemini-2.0-flash',
'gemini-2.0-flash-001',
'gemini-2.0-pro-exp-02-05',
'gemini-2.0-flash-lite-preview-02-05',
'gemini-1.5-flash',
'gemini-1.5-pro',
'gemini-1.5-pro-002',
'gemini-1.5-flash-8b',
'gemini-2.0-flash-thinking-exp-01-21']Gemini has several types of models and only some of those work with VertexAI client. Input wise, essentially all models support images and text, and since Gemini 1.5 all (except gemini-2.0-flash-thinking-exp-01-21) support audio and video input.
Right now only imagen models support non text output, but image and audio outputs are coming to Gemini 2 “soon”.
Main models support function calling, structured output, code executions (with a few exception on the preview models) and all models support streaming and system prompts.
Although it would be nice to provide a uniform library interface with cosette and claudette, it’s probably a bit too complex for the moment.
from IPython.display import MarkdownGEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", None)
Markdown("* " + "* ".join([f"`{model.name}`: {model.description}\n" for model in genai.Client(api_key=GEMINI_API_KEY).models.list()]))models/chat-bison-001: A legacy text-only model optimized for chat conversationsmodels/text-bison-001: A legacy model that understands text and generates text as an outputmodels/embedding-gecko-001: Obtain a distributed representation of a text.models/gemini-1.0-pro-latest: The original Gemini 1.0 Pro model. This model will be discontinued on February 15th, 2025. Move to a newer Gemini version.models/gemini-1.0-pro: The best model for scaling across a wide range of tasksmodels/gemini-pro: The best model for scaling across a wide range of tasksmodels/gemini-1.0-pro-001: The original Gemini 1.0 Pro model version that supports tuning. Gemini 1.0 Pro will be discontinued on February 15th, 2025. Move to a newer Gemini version.models/gemini-1.0-pro-vision-latest: The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.models/gemini-pro-vision: The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.models/gemini-1.5-pro-latest: Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.models/gemini-1.5-pro-001: Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.models/gemini-1.5-pro-002: Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in September of 2024.models/gemini-1.5-pro: Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.models/gemini-1.5-flash-latest: Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.models/gemini-1.5-flash-001: Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.models/gemini-1.5-flash-001-tuning: Version of Gemini 1.5 Flash that supports tuning, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.models/gemini-1.5-flash: Alias that points to the most recent stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.models/gemini-1.5-flash-002: Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in September of 2024.models/gemini-1.5-flash-8b: Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.models/gemini-1.5-flash-8b-001: Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.models/gemini-1.5-flash-8b-latest: Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.models/gemini-1.5-flash-8b-exp-0827: Experimental release (August 27th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).models/gemini-1.5-flash-8b-exp-0924: Experimental release (September 24th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).models/gemini-2.0-flash-exp: Gemini 2.0 Flash Experimentalmodels/gemini-2.0-flash: Gemini 2.0 Flashmodels/gemini-2.0-flash-001: Stable version of Gemini 2.0 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in January of 2025.models/gemini-2.0-flash-lite-preview: Preview release (February 5th, 2025) of Gemini 2.0 Flash Litemodels/gemini-2.0-flash-lite-preview-02-05: Preview release (February 5th, 2025) of Gemini 2.0 Flash Litemodels/gemini-2.0-pro-exp: Experimental release (February 5th, 2025) of Gemini 2.0 Promodels/gemini-2.0-pro-exp-02-05: Experimental release (February 5th, 2025) of Gemini 2.0 Promodels/gemini-exp-1206: Experimental release (February 5th, 2025) of Gemini 2.0 Promodels/gemini-2.0-flash-thinking-exp-01-21: Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinkingmodels/gemini-2.0-flash-thinking-exp: Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinkingmodels/gemini-2.0-flash-thinking-exp-1219: Gemini 2.0 Flash Thinking Experimentalmodels/learnlm-1.5-pro-experimental: Alias that points to the most recent stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.models/embedding-001: Obtain a distributed representation of a text.models/text-embedding-004: Obtain a distributed representation of a text.models/aqa: Model trained to return answers to questions that are grounded in provided sources, along with estimating answerable probability.models/imagen-3.0-generate-002: Vertex served Imagen 3.0 002 modelThe Genai API exposes way more models than what we have included, but most of those are outdated, the list is not properly maintained and not all models behave in the same way. Although all of them will still be available, it’s is fine to just restrict to a few select ones.
c = genai.Client(api_key=GEMINI_API_KEY)This how the Gemini SDK gives access to the API. The client itself, in particular has a number of subclients/methods that give access to the different endpoint and functionalities. The main ones we are interested in are models, that is the main interface with all the models, and chat which essentially wraps the former with a few convenience functionalities for handling message history.
model = models[0]
model'gemini-2.0-flash'r = c.models.generate_content(model=model, contents="Hi Gemini! Are you ready to work?")
rGenerateContentResponse(candidates=[Candidate(content=Content(parts=[Part(video_metadata=None, thought=None, code_execution_result=None, executable_code=None, file_data=None, function_call=None, function_response=None, inline_data=None, text='Yes, I am ready to work! What can I do for you?\n')], role='model'), citation_metadata=None, finish_message=None, token_count=None, avg_logprobs=-0.36470311880111694, finish_reason=<FinishReason.STOP: 'STOP'>, grounding_metadata=None, index=None, logprobs_result=None, safety_ratings=None)], model_version='gemini-2.0-flash', prompt_feedback=None, usage_metadata=GenerateContentResponseUsageMetadata(cached_content_token_count=None, candidates_token_count=16, prompt_token_count=9, total_token_count=25), automatic_function_calling_history=[], parsed=None)r.to_json_dict(){'candidates': [{'content': {'parts': [{'text': 'Yes, I am ready to work! What can I do for you?\n'}],
'role': 'model'},
'avg_logprobs': -0.36470311880111694,
'finish_reason': 'STOP'}],
'model_version': 'gemini-2.0-flash',
'usage_metadata': {'candidates_token_count': 16,
'prompt_token_count': 9,
'total_token_count': 25},
'automatic_function_calling_history': []}print(r.text)Yes, I am ready to work! What can I do for you?
In typical Google fashion (they really like their protobufs), the response is a nested mess of pydantic models. Luckily they all have a few convenience methods to make everything a bit more accessible.
help(genai._common.BaseModel)Help on class BaseModel in module google.genai._common:
class BaseModel(pydantic.main.BaseModel)
| BaseModel() -> None
|
| Method resolution order:
| BaseModel
| pydantic.main.BaseModel
| builtins.object
|
| Methods defined here:
|
| to_json_dict(self) -> dict[str, object]
|
| ----------------------------------------------------------------------
| Data descriptors defined here:
|
| __weakref__
| list of weak references to the object
|
| ----------------------------------------------------------------------
| Data and other attributes defined here:
|
| __abstractmethods__ = frozenset()
|
| __annotations__ = {}
|
| __class_vars__ = set()
|
| __private_attributes__ = {}
|
| __pydantic_complete__ = True
|
| __pydantic_computed_fields__ = {}
|
| __pydantic_core_schema__ = {'cls': <class 'google.genai._common.BaseMo...
|
| __pydantic_custom_init__ = False
|
| __pydantic_decorators__ = DecoratorInfos(validators={}, field_validato...
|
| __pydantic_fields__ = {}
|
| __pydantic_generic_metadata__ = {'args': (), 'origin': None, 'paramete...
|
| __pydantic_parent_namespace__ = None
|
| __pydantic_post_init__ = None
|
| __pydantic_serializer__ = SchemaSerializer(serializer=Model(
| Model...
|
| __pydantic_validator__ = SchemaValidator(title="BaseModel", validator=...
|
| __signature__ = <Signature () -> None>
|
| model_config = {'alias_generator': <function to_camel>, 'arbitrary_typ...
|
| ----------------------------------------------------------------------
| Methods inherited from pydantic.main.BaseModel:
|
| __copy__(self) -> 'Self'
| Returns a shallow copy of the model.
|
| __deepcopy__(self, memo: 'dict[int, Any] | None' = None) -> 'Self'
| Returns a deep copy of the model.
|
| __delattr__(self, item: 'str') -> 'Any'
| Implement delattr(self, name).
|
| __eq__(self, other: 'Any') -> 'bool'
| Return self==value.
|
| __getattr__(self, item: 'str') -> 'Any'
|
| __getstate__(self) -> 'dict[Any, Any]'
| Helper for pickle.
|
| __init__(self, /, **data: 'Any') -> 'None'
| Create a new model by parsing and validating input data from keyword arguments.
|
| Raises [`ValidationError`][pydantic_core.ValidationError] if the input data cannot be
| validated to form a valid model.
|
| `self` is explicitly positional-only to allow `self` as a field name.
|
| __iter__(self) -> 'TupleGenerator'
| So `dict(model)` works.
|
| __pretty__(self, fmt: 'typing.Callable[[Any], Any]', **kwargs: 'Any') -> 'typing.Generator[Any, None, None]' from pydantic._internal._repr.Representation
| Used by devtools (https://python-devtools.helpmanual.io/) to pretty print objects.
|
| __replace__(self, **changes: 'Any') -> 'Self'
| # Because we make use of `@dataclass_transform()`, `__replace__` is already synthesized by
| # type checkers, so we define the implementation in this `if not TYPE_CHECKING:` block:
|
| __repr__(self) -> 'str'
| Return repr(self).
|
| __repr_args__(self) -> '_repr.ReprArgs'
|
| __repr_name__(self) -> 'str' from pydantic._internal._repr.Representation
| Name of the instance's class, used in __repr__.
|
| __repr_recursion__(self, object: 'Any') -> 'str' from pydantic._internal._repr.Representation
| Returns the string representation of a recursive object.
|
| __repr_str__(self, join_str: 'str') -> 'str' from pydantic._internal._repr.Representation
|
| __rich_repr__(self) -> 'RichReprResult' from pydantic._internal._repr.Representation
| Used by Rich (https://rich.readthedocs.io/en/stable/pretty.html) to pretty print objects.
|
| __setattr__(self, name: 'str', value: 'Any') -> 'None'
| Implement setattr(self, name, value).
|
| __setstate__(self, state: 'dict[Any, Any]') -> 'None'
|
| __str__(self) -> 'str'
| Return str(self).
|
| copy(self, *, include: 'AbstractSetIntStr | MappingIntStrAny | None' = None, exclude: 'AbstractSetIntStr | MappingIntStrAny | None' = None, update: 'Dict[str, Any] | None' = None, deep: 'bool' = False) -> 'Self'
| Returns a copy of the model.
|
| !!! warning "Deprecated"
| This method is now deprecated; use `model_copy` instead.
|
| If you need `include` or `exclude`, use:
|
| ```python {test="skip" lint="skip"}
| data = self.model_dump(include=include, exclude=exclude, round_trip=True)
| data = {**data, **(update or {})}
| copied = self.model_validate(data)
| ```
|
| Args:
| include: Optional set or mapping specifying which fields to include in the copied model.
| exclude: Optional set or mapping specifying which fields to exclude in the copied model.
| update: Optional dictionary of field-value pairs to override field values in the copied model.
| deep: If True, the values of fields that are Pydantic models will be deep-copied.
|
| Returns:
| A copy of the model with included, excluded and updated fields as specified.
|
| dict(self, *, include: 'IncEx | None' = None, exclude: 'IncEx | None' = None, by_alias: 'bool' = False, exclude_unset: 'bool' = False, exclude_defaults: 'bool' = False, exclude_none: 'bool' = False) -> 'Dict[str, Any]'
|
| json(self, *, include: 'IncEx | None' = None, exclude: 'IncEx | None' = None, by_alias: 'bool' = False, exclude_unset: 'bool' = False, exclude_defaults: 'bool' = False, exclude_none: 'bool' = False, encoder: 'Callable[[Any], Any] | None' = PydanticUndefined, models_as_dict: 'bool' = PydanticUndefined, **dumps_kwargs: 'Any') -> 'str'
|
| model_copy(self, *, update: 'Mapping[str, Any] | None' = None, deep: 'bool' = False) -> 'Self'
| Usage docs: https://docs.pydantic.dev/2.10/concepts/serialization/#model_copy
|
| Returns a copy of the model.
|
| Args:
| update: Values to change/add in the new model. Note: the data is not validated
| before creating the new model. You should trust this data.
| deep: Set to `True` to make a deep copy of the model.
|
| Returns:
| New model instance.
|
| model_dump(self, *, mode: "Literal['json', 'python'] | str" = 'python', include: 'IncEx | None' = None, exclude: 'IncEx | None' = None, context: 'Any | None' = None, by_alias: 'bool' = False, exclude_unset: 'bool' = False, exclude_defaults: 'bool' = False, exclude_none: 'bool' = False, round_trip: 'bool' = False, warnings: "bool | Literal['none', 'warn', 'error']" = True, serialize_as_any: 'bool' = False) -> 'dict[str, Any]'
| Usage docs: https://docs.pydantic.dev/2.10/concepts/serialization/#modelmodel_dump
|
| Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.
|
| Args:
| mode: The mode in which `to_python` should run.
| If mode is 'json', the output will only contain JSON serializable types.
| If mode is 'python', the output may contain non-JSON-serializable Python objects.
| include: A set of fields to include in the output.
| exclude: A set of fields to exclude from the output.
| context: Additional context to pass to the serializer.
| by_alias: Whether to use the field's alias in the dictionary key if defined.
| exclude_unset: Whether to exclude fields that have not been explicitly set.
| exclude_defaults: Whether to exclude fields that are set to their default value.
| exclude_none: Whether to exclude fields that have a value of `None`.
| round_trip: If True, dumped values should be valid as input for non-idempotent types such as Json[T].
| warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors,
| "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
| serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
|
| Returns:
| A dictionary representation of the model.
|
| model_dump_json(self, *, indent: 'int | None' = None, include: 'IncEx | None' = None, exclude: 'IncEx | None' = None, context: 'Any | None' = None, by_alias: 'bool' = False, exclude_unset: 'bool' = False, exclude_defaults: 'bool' = False, exclude_none: 'bool' = False, round_trip: 'bool' = False, warnings: "bool | Literal['none', 'warn', 'error']" = True, serialize_as_any: 'bool' = False) -> 'str'
| Usage docs: https://docs.pydantic.dev/2.10/concepts/serialization/#modelmodel_dump_json
|
| Generates a JSON representation of the model using Pydantic's `to_json` method.
|
| Args:
| indent: Indentation to use in the JSON output. If None is passed, the output will be compact.
| include: Field(s) to include in the JSON output.
| exclude: Field(s) to exclude from the JSON output.
| context: Additional context to pass to the serializer.
| by_alias: Whether to serialize using field aliases.
| exclude_unset: Whether to exclude fields that have not been explicitly set.
| exclude_defaults: Whether to exclude fields that are set to their default value.
| exclude_none: Whether to exclude fields that have a value of `None`.
| round_trip: If True, dumped values should be valid as input for non-idempotent types such as Json[T].
| warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors,
| "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
| serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
|
| Returns:
| A JSON string representation of the model.
|
| model_post_init(self, _BaseModel__context: 'Any') -> 'None'
| Override this method to perform additional initialization after `__init__` and `model_construct`.
| This is useful if you want to do some validation that requires the entire model to be initialized.
|
| ----------------------------------------------------------------------
| Class methods inherited from pydantic.main.BaseModel:
|
| __class_getitem__(typevar_values: 'type[Any] | tuple[type[Any], ...]') -> 'type[BaseModel] | _forward_ref.PydanticRecursiveRef'
|
| __get_pydantic_core_schema__(source: 'type[BaseModel]', handler: 'GetCoreSchemaHandler', /) -> 'CoreSchema'
| Hook into generating the model's CoreSchema.
|
| Args:
| source: The class we are generating a schema for.
| This will generally be the same as the `cls` argument if this is a classmethod.
| handler: A callable that calls into Pydantic's internal CoreSchema generation logic.
|
| Returns:
| A `pydantic-core` `CoreSchema`.
|
| __get_pydantic_json_schema__(core_schema: 'CoreSchema', handler: 'GetJsonSchemaHandler', /) -> 'JsonSchemaValue'
| Hook into generating the model's JSON schema.
|
| Args:
| core_schema: A `pydantic-core` CoreSchema.
| You can ignore this argument and call the handler with a new CoreSchema,
| wrap this CoreSchema (`{'type': 'nullable', 'schema': current_schema}`),
| or just call the handler with the original schema.
| handler: Call into Pydantic's internal JSON schema generation.
| This will raise a `pydantic.errors.PydanticInvalidForJsonSchema` if JSON schema
| generation fails.
| Since this gets called by `BaseModel.model_json_schema` you can override the
| `schema_generator` argument to that function to change JSON schema generation globally
| for a type.
|
| Returns:
| A JSON schema, as a Python object.
|
| __pydantic_init_subclass__(**kwargs: 'Any') -> 'None'
| This is intended to behave just like `__init_subclass__`, but is called by `ModelMetaclass`
| only after the class is actually fully initialized. In particular, attributes like `model_fields` will
| be present when this is called.
|
| This is necessary because `__init_subclass__` will always be called by `type.__new__`,
| and it would require a prohibitively large refactor to the `ModelMetaclass` to ensure that
| `type.__new__` was called in such a manner that the class would already be sufficiently initialized.
|
| This will receive the same `kwargs` that would be passed to the standard `__init_subclass__`, namely,
| any kwargs passed to the class definition that aren't used internally by pydantic.
|
| Args:
| **kwargs: Any keyword arguments passed to the class definition that aren't used internally
| by pydantic.
|
| construct(_fields_set: 'set[str] | None' = None, **values: 'Any') -> 'Self'
|
| from_orm(obj: 'Any') -> 'Self'
|
| model_construct(_fields_set: 'set[str] | None' = None, **values: 'Any') -> 'Self'
| Creates a new instance of the `Model` class with validated data.
|
| Creates a new model setting `__dict__` and `__pydantic_fields_set__` from trusted or pre-validated data.
| Default values are respected, but no other validation is performed.
|
| !!! note
| `model_construct()` generally respects the `model_config.extra` setting on the provided model.
| That is, if `model_config.extra == 'allow'`, then all extra passed values are added to the model instance's `__dict__`
| and `__pydantic_extra__` fields. If `model_config.extra == 'ignore'` (the default), then all extra passed values are ignored.
| Because no validation is performed with a call to `model_construct()`, having `model_config.extra == 'forbid'` does not result in
| an error if extra values are passed, but they will be ignored.
|
| Args:
| _fields_set: A set of field names that were originally explicitly set during instantiation. If provided,
| this is directly used for the [`model_fields_set`][pydantic.BaseModel.model_fields_set] attribute.
| Otherwise, the field names from the `values` argument will be used.
| values: Trusted or pre-validated data dictionary.
|
| Returns:
| A new instance of the `Model` class with validated data.
|
| model_json_schema(by_alias: 'bool' = True, ref_template: 'str' = '#/$defs/{model}', schema_generator: 'type[GenerateJsonSchema]' = <class 'pydantic.json_schema.GenerateJsonSchema'>, mode: 'JsonSchemaMode' = 'validation') -> 'dict[str, Any]'
| Generates a JSON schema for a model class.
|
| Args:
| by_alias: Whether to use attribute aliases or not.
| ref_template: The reference template.
| schema_generator: To override the logic used to generate the JSON schema, as a subclass of
| `GenerateJsonSchema` with your desired modifications
| mode: The mode in which to generate the schema.
|
| Returns:
| The JSON schema for the given model class.
|
| model_parametrized_name(params: 'tuple[type[Any], ...]') -> 'str'
| Compute the class name for parametrizations of generic classes.
|
| This method can be overridden to achieve a custom naming scheme for generic BaseModels.
|
| Args:
| params: Tuple of types of the class. Given a generic class
| `Model` with 2 type variables and a concrete model `Model[str, int]`,
| the value `(str, int)` would be passed to `params`.
|
| Returns:
| String representing the new class where `params` are passed to `cls` as type variables.
|
| Raises:
| TypeError: Raised when trying to generate concrete names for non-generic models.
|
| model_rebuild(*, force: 'bool' = False, raise_errors: 'bool' = True, _parent_namespace_depth: 'int' = 2, _types_namespace: 'MappingNamespace | None' = None) -> 'bool | None'
| Try to rebuild the pydantic-core schema for the model.
|
| This may be necessary when one of the annotations is a ForwardRef which could not be resolved during
| the initial attempt to build the schema, and automatic rebuilding fails.
|
| Args:
| force: Whether to force the rebuilding of the model schema, defaults to `False`.
| raise_errors: Whether to raise errors, defaults to `True`.
| _parent_namespace_depth: The depth level of the parent namespace, defaults to 2.
| _types_namespace: The types namespace, defaults to `None`.
|
| Returns:
| Returns `None` if the schema is already "complete" and rebuilding was not required.
| If rebuilding _was_ required, returns `True` if rebuilding was successful, otherwise `False`.
|
| model_validate(obj: 'Any', *, strict: 'bool | None' = None, from_attributes: 'bool | None' = None, context: 'Any | None' = None) -> 'Self'
| Validate a pydantic model instance.
|
| Args:
| obj: The object to validate.
| strict: Whether to enforce types strictly.
| from_attributes: Whether to extract data from object attributes.
| context: Additional context to pass to the validator.
|
| Raises:
| ValidationError: If the object could not be validated.
|
| Returns:
| The validated model instance.
|
| model_validate_json(json_data: 'str | bytes | bytearray', *, strict: 'bool | None' = None, context: 'Any | None' = None) -> 'Self'
| Usage docs: https://docs.pydantic.dev/2.10/concepts/json/#json-parsing
|
| Validate the given JSON data against the Pydantic model.
|
| Args:
| json_data: The JSON data to validate.
| strict: Whether to enforce types strictly.
| context: Extra variables to pass to the validator.
|
| Returns:
| The validated Pydantic model.
|
| Raises:
| ValidationError: If `json_data` is not a JSON string or the object could not be validated.
|
| model_validate_strings(obj: 'Any', *, strict: 'bool | None' = None, context: 'Any | None' = None) -> 'Self'
| Validate the given object with string data against the Pydantic model.
|
| Args:
| obj: The object containing string data to validate.
| strict: Whether to enforce types strictly.
| context: Extra variables to pass to the validator.
|
| Returns:
| The validated Pydantic model.
|
| parse_file(path: 'str | Path', *, content_type: 'str | None' = None, encoding: 'str' = 'utf8', proto: 'DeprecatedParseProtocol | None' = None, allow_pickle: 'bool' = False) -> 'Self'
|
| parse_obj(obj: 'Any') -> 'Self'
|
| parse_raw(b: 'str | bytes', *, content_type: 'str | None' = None, encoding: 'str' = 'utf8', proto: 'DeprecatedParseProtocol | None' = None, allow_pickle: 'bool' = False) -> 'Self'
|
| schema(by_alias: 'bool' = True, ref_template: 'str' = '#/$defs/{model}') -> 'Dict[str, Any]'
|
| schema_json(*, by_alias: 'bool' = True, ref_template: 'str' = '#/$defs/{model}', **dumps_kwargs: 'Any') -> 'str'
|
| update_forward_refs(**localns: 'Any') -> 'None'
|
| validate(value: 'Any') -> 'Self'
|
| ----------------------------------------------------------------------
| Readonly properties inherited from pydantic.main.BaseModel:
|
| __fields_set__
|
| model_computed_fields
| Get metadata about the computed fields defined on the model.
|
| Deprecation warning: you should be getting this information from the model class, not from an instance.
| In V3, this property will be removed from the `BaseModel` class.
|
| Returns:
| A mapping of computed field names to [`ComputedFieldInfo`][pydantic.fields.ComputedFieldInfo] objects.
|
| model_extra
| Get extra fields set during validation.
|
| Returns:
| A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`.
|
| model_fields
| Get metadata about the fields defined on the model.
|
| Deprecation warning: you should be getting this information from the model class, not from an instance.
| In V3, this property will be removed from the `BaseModel` class.
|
| Returns:
| A mapping of field names to [`FieldInfo`][pydantic.fields.FieldInfo] objects.
|
| model_fields_set
| Returns the set of fields that have been explicitly set on this model instance.
|
| Returns:
| A set of strings representing the fields that have been set,
| i.e. that were not filled from defaults.
|
| ----------------------------------------------------------------------
| Data descriptors inherited from pydantic.main.BaseModel:
|
| __dict__
| dictionary for instance variables
|
| __pydantic_extra__
|
| __pydantic_fields_set__
|
| __pydantic_private__
|
| ----------------------------------------------------------------------
| Data and other attributes inherited from pydantic.main.BaseModel:
|
| __hash__ = None
|
| __pydantic_root_model__ = False
All the models in genai.types that are then used to cast the interaction back and forth with the API are subclasses of the genai._common.BaseModel. Although this being a “private” module make it less than ideal, it will be useful to quickly monkey patch a better representation of the output.
types.GenerateContentConfigDict??Init signature: types.GenerateContentConfigDict(self, /, *args, **kwargs) Source: class GenerateContentConfigDict(TypedDict, total=False): """Optional model configuration parameters. For more information, see `Content generation parameters <https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/content-generation-parameters>`_. """ http_options: Optional[HttpOptionsDict] """Used to override HTTP request options.""" system_instruction: Optional[ContentUnionDict] """Instructions for the model to steer it toward better performance. For example, "Answer as concisely as possible" or "Don't use technical terms in your response". """ temperature: Optional[float] """Value that controls the degree of randomness in token selection. Lower temperatures are good for prompts that require a less open-ended or creative response, while higher temperatures can lead to more diverse or creative results. """ top_p: Optional[float] """Tokens are selected from the most to least probable until the sum of their probabilities equals this value. Use a lower value for less random responses and a higher value for more random responses. """ top_k: Optional[float] """For each token selection step, the ``top_k`` tokens with the highest probabilities are sampled. Then tokens are further filtered based on ``top_p`` with the final token selected using temperature sampling. Use a lower number for less random responses and a higher number for more random responses. """ candidate_count: Optional[int] """Number of response variations to return. """ max_output_tokens: Optional[int] """Maximum number of tokens that can be generated in the response. """ stop_sequences: Optional[list[str]] """List of strings that tells the model to stop generating text if one of the strings is encountered in the response. """ response_logprobs: Optional[bool] """Whether to return the log probabilities of the tokens that were chosen by the model at each step. """ logprobs: Optional[int] """Number of top candidate tokens to return the log probabilities for at each generation step. """ presence_penalty: Optional[float] """Positive values penalize tokens that already appear in the generated text, increasing the probability of generating more diverse content. """ frequency_penalty: Optional[float] """Positive values penalize tokens that repeatedly appear in the generated text, increasing the probability of generating more diverse content. """ seed: Optional[int] """When ``seed`` is fixed to a specific number, the model makes a best effort to provide the same response for repeated requests. By default, a random number is used. """ response_mime_type: Optional[str] """Output response media type of the generated candidate text. """ response_schema: Optional[SchemaUnionDict] """Schema that the generated candidate text must adhere to. """ routing_config: Optional[GenerationConfigRoutingConfigDict] """Configuration for model router requests. """ safety_settings: Optional[list[SafetySettingDict]] """Safety settings in the request to block unsafe content in the response. """ tools: Optional[ToolListUnionDict] """Code that enables the system to interact with external systems to perform an action outside of the knowledge and scope of the model. """ tool_config: Optional[ToolConfigDict] """Associates model output to a specific function call. """ labels: Optional[dict[str, str]] """Labels with user-defined metadata to break down billed charges.""" cached_content: Optional[str] """Resource name of a context cache that can be used in subsequent requests. """ response_modalities: Optional[list[str]] """The requested modalities of the response. Represents the set of modalities that the model can return. """ media_resolution: Optional[MediaResolution] """If specified, the media resolution specified will be used. """ speech_config: Optional[SpeechConfigUnionDict] """The speech generation configuration. """ audio_timestamp: Optional[bool] """If enabled, audio timestamp will be included in the request to the model. """ automatic_function_calling: Optional[AutomaticFunctionCallingConfigDict] """The configuration for automatic function calling. """ thinking_config: Optional[ThinkingConfigDict] """The thinking features configuration. """ File: ~/mambaforge/envs/prototypes/lib/python3.12/site-packages/google/genai/types.py Type: _TypedDictMeta Subclasses:
All the generation paramters can be passed as a dictionary. The available parameters are defined by the GenerateContentConfigDict model (which is just a snake case/dictionary conversion of GenerateContentConfig). This include things like temperature and top_k/top_p value as well as stopping sequences and system prompt (which is actually passed at each generation)
sr = c.models.generate_content(model=model,
contents="Count from 1 to 5 and add a write a different animal after each number",
config={"stop_sequences": ["4"]})
print(sr.text)Okay, here we go!
1 - Cat
2 - Dog
3 - Bird
spr = c.models.generate_content(model=model,
contents="Count from 1 to 5 and add a write a different animal after each number",
config={"system_instruction": "Always talk in Spanish"})
print(spr.text)¡Por supuesto! Aquí tienes:
1. Uno... Perro
2. Dos... Gato
3. Tres... Elefante
4. Cuatro... León
5. Cinco... Ballena
for chunk in c.models.generate_content_stream(model=model, contents="Write a small poem about the hardships of being a cocker spaniel"):
print(chunk.text, end='')With ears so long, a tripping hazard,
Through muddy fields, a muddy blaggard.
A feathered tail, a joyful wag,
Yet burrs collect, a constant snag.
My soulful eyes, they plead and yearn,
For walks and treats, a love to earn.
But oh, the groomer, shears so bright,
A cocker's life, a fluffy fight.type(chunk)google.genai.types.GenerateContentResponseStreaming content is handled with a separate method of models but everything else stays the same.
mr = c.models.generate_images(
model = imagen_models[0],
prompt = "A roman mosaic of a boxing match between a feathered dinosaur and a cocker spaniel,\
refereed by Sasquatch, in front of a crowd of cheering otters.",
config = {"number_of_images": 2}
)type(mr)google.genai.types.GenerateImagesResponseCurrently the only multimedia output supported is image generation via imagen models (although audio and image generation with Gemini 2.0 is currently being tested in restricted availability). This uses a separate method, and returns a different type of response. The possible options are defined in the pydantic model but not all of them are actually available (for example, the enhance_prompt option does not work with the Gemini API).
for genim in mr.generated_images:
genim.image.show()
mr.generated_images[0].image.save('match.png')As usual, the response is a convoluted nested mess of pydantic models. The images are returned as a wrapper of a PIL.Image which gives access to a few convenience function for displaying and saving. Right now jpeg is the only possible output format.
r.text'Yes, I am ready to work! What can I do for you?\n'r.to_json_dict(){'candidates': [{'content': {'parts': [{'text': 'Yes, I am ready to work! What can I do for you?\n'}],
'role': 'model'},
'avg_logprobs': -0.36470311880111694,
'finish_reason': 'STOP'}],
'model_version': 'gemini-2.0-flash',
'usage_metadata': {'candidates_token_count': 16,
'prompt_token_count': 9,
'total_token_count': 25},
'automatic_function_calling_history': []}r.model_fields_set{'automatic_function_calling_history',
'candidates',
'model_version',
'usage_metadata'}type(r.usage_metadata), type(r.candidates), type(r.model_version)(google.genai.types.GenerateContentResponseUsageMetadata, list, str)We want to recursively navigate the nested tree of submodels and attributes. Each model in genai.types has three types of attributes:
genai.types module (like usage_metadata)candidates)model_version)We could extract the attributes of a model from the class itself, but to avoid cluttering the output, we can use the model_fields_set property to avoid attributes that are not set in the model instance.
get_repr (m, lab='')
Recurisvely fetch the markdown representation of genai.types fields, wrapping lists into <details> blocks
def get_repr(m, lab=""):
"""Recurisvely fetch the markdown representation of genai.types fields, wrapping lists into `<details>` blocks"""
if hasattr(m, '_repr_markdown_'): return m._repr_markdown_()
if is_listy(m): return "\n".join([f"<details open='true'><summary>{lab}[{i}]</summary>{get_repr(li)}</details>" for i, li in enumerate(m)])
if isinstance(m, dict): return "<ul>" + "\n".join([f"<li><b>{i}</b>: {get_repr(li, i)}</li>" for i, li in m.items()]) + "</ul>"
if isinstance(m, bytes): return m[:10] + b'...'
return str(m)Markdown(get_repr([["A", "B", "C"], "b", "c", {"x": 2, "y": {"as": "sa"}}], "ex"))The basic recursive loop is in place: we can handle genai.types (via their _repr_markdown_ methods), strings and lists. The handling of bytes is to avoid polluting (or crashing) the representation with a huge list of charachters in case of a multimodal response.
det_repr (m)
def det_repr(m): return "<ul>" + "".join(f"<li><code>{d}</code>: {get_repr(getattr(m, d), d)}</li>" for d in m.model_fields_set) + "</ul>"Markdown(det_repr(r.usage_metadata))candidates_token_count: 16
total_token_count: 25
prompt_token_count: 9
Wrapping the details in a list makes for a cleaner and more readable look.
@patch
def _repr_markdown_(self: genai._common.BaseModel):
return det_repr(self)rusage_metadata:
candidates_token_count: 16
total_token_count: 25
prompt_token_count: 9
model_version: gemini-2.0-flash
candidates:
avg_logprobs: -0.36470311880111694
content:
parts:
text: Yes, I am ready to work! What can I do for you?
role: model
finish_reason: FinishReason.STOP
automatic_function_calling_history:
By using fastcore’s patch on the _common.BaseModel we have made sure that all the models in genai.types have a nice consistent markdown representation. We can now refine the representation for some of the types.
@patch
def _repr_markdown_(self: genai.types.GenerateContentResponse):
c = None
try:
c = self.text.replace("\n", "<br />")
except ValueError as e:
calls = (f"<code>{call.name}({', '.join([f'{a}={v}' for a, v in call.args.items()])})</code>" for call in self.function_calls)
calls_repr = '\n'.join(f'<li>{c}</li>' for c in calls)
c = f"<ul>{calls_repr}</ul>"
dets = det_repr(self)
return f"""{c}\n<details>{dets}</details>"""rusage_metadata: Cached: 0; In: 9; Out: 16; Total: 25
model_version: gemini-2.0-flash
candidates:
avg_logprobs: -0.36470311880111694
content:
parts:
text: Yes, I am ready to work! What can I do for you?
role: model
finish_reason: FinishReason.STOP
automatic_function_calling_history:
mrgenerated_images:
image:
mime_type: image/png
image_bytes: b’89PNG1a…’
image:
mime_type: image/png
image_bytes: b’89PNG1a…’
Image.html ()
@patch(as_prop=True)
def html(self: types.Image):
b64 = base64.b64encode(self.image_bytes).decode("utf-8")
return f'<img src="data:{self.mime_type};base64,{b64}" />'
@patch
def _repr_markdown_(self: types.Image):
return f"""<div style="width: 100px; height: auto;">{self.html}</div>
<details>
{det_repr(self)}
</details>"""mr.generated_images[0].image
mime_type: image/png
image_bytes: b’89PNG1a…’
GenerateImagesResponse.img ()
@patch(as_prop=True)
def img(self: types.GenerateImagesResponse):
return self.generated_images[0].image._pil_image
@patch
def _repr_markdown_(self: types.GenerateImagesResponse):
N = len(self.generated_images)
cols = min(N, 4)
rows = math.ceil(N / 4)
ims = "".join([f"""<div style="display: grid;
width: 100%;
max-width: 1000px;
height: auto;
margin: 0 auto;
grid-template-columns: {rows}fr;
grid-template-rows: 1ft;
">{gim.image.html}</div>"""
for gim in self.generated_images])
i = f"""
<div style="display: grid;
gap: 4px;
width: 100%;
height: auto;
max-width: 1000px;
margin: 0 auto;
padding: 4px;
grid-template-columns: repeat({cols}, 1fr);
grid-template-rows: repeat({rows}, 1fr);
">
{ims}
</div>
"""
return f"""{i}
<details>
{det_repr(self)}
</details>
"""mr.img
mr
generated_images:
image:
